
# Create a temporary directory to hold notebook output, and change the working directory to that directory.
from tempfile import TemporaryDirectory
import os
import shutil
data_dir = TemporaryDirectory()

os.chdir(data_dir.name)


import os
if 'CRDS_CACHE_TYPE' in os.environ:
    if os.environ['CRDS_CACHE_TYPE'] == 'local':
        os.environ['CRDS_PATH'] = os.path.join(os.environ['HOME'], 'crds', 'cache')
    elif os.path.isdir(os.environ['CRDS_CACHE_TYPE']):
        os.environ['CRDS_PATH'] = os.environ['CRDS_CACHE_TYPE']
print('CRDS cache location: {}'.format(os.environ['CRDS_PATH']))

CRDS cache location: /grp/crds/cache


import warnings
import psutil
from astropy.io import fits

# Only print a DeprecationWarning the first time it shows up, not every time.
with warnings.catch_warnings():
    warnings.simplefilter("once", category=DeprecationWarning)
    import jwst
    from jwst.pipeline.calwebb_detector1 import Detector1Pipeline
    from jwst.assign_wcs.assign_wcs_step import AssignWcsStep
    from jwst.msaflagopen.msaflagopen_step import MSAFlagOpenStep
    from jwst import datamodels

# The latest version of NPTT is installed in the requirements text file at:
# /jwst_validation_notebooks/environment.yml

# import NPTT
import nirspec_pipe_testing_tool as nptt

# To get data from Artifactory
from ci_watson.artifactory_helpers import get_bigdata


# Make sure that the version used is the right one

pipeline_version = jwst.__version__
nptt_version = nptt.__version__

print("Using jwst pipeline version: ", pipeline_version)
print("Using NPTT version: ", nptt_version)

Using jwst pipeline version:  1.7.2
Using NPTT version:  2.0.1


testing_data = {                
                'mos_g140m_f100lp':{
                                  'uncal_file_nrs1': 'mos_g140m_line1_NRS1_uncal.fits',
                                  'uncal_file_nrs2': 'mos_g140m_line1_NRS2_uncal.fits',  
                                  'msa_shutter_config': 'V8460001000101_msa.fits' }
               }


# define function to pull data from Artifactory
def get_artifactory_file(data_set_dict, detector):
    """This function creates a list with all the files needed per detector to run the test.
    Args:
        data_set_dict: dictionary, contains inputs for a specific mode and configuration
        detector: string, either nrs1 or nrs2
    Returns:
        data: list, contains all files needed to run test
    """
    files2obtain = ['uncal_file_nrs1', 'msa_shutter_config']
    data = []
    for file in files2obtain:
        data_file = None
        try: 
            if '_nrs' in file and '2' in detector:
                file = file.replace('_nrs1', '_nrs2')

            data_file = get_bigdata('jwst_validation_notebooks',
                                         'validation_data',
                                         'nirspec_data', 
                                         data_set_dict[file])
        except TypeError:
            data.append(None)
            continue

        data.append(data_file)

    return data


# Set common NPTT switches for this test

# accepted threshold difference with respect to benchmark files
msa_flagging_threshold = 99.5

# other NPTT variables
stellarity = None
operability_ref = None
source_type = None
save_figs = False
show_figs = True
debug = False
verbose = False

# Get the data
results_dict = {}
detectors = ['nrs1', 'nrs2']
for mode_config, data_set_dict in testing_data.items():
    for det in detectors:
        print('Testing files for detector: ', det)
        data = get_artifactory_file(data_set_dict, det)
        uncal_file, msa_shutter_config = data
        print('Working with uncal_file: ', uncal_file)
        uncal_basename = os.path.basename(uncal_file)
                
        # Run the stage 1 pipeline 
        rate_object = Detector1Pipeline.call(uncal_file)
        # Make sure the MSA shutter configuration file is set up correctly
        if msa_shutter_config is not None:
            rate_object.meta.instrument.msa_metadata_file = msa_shutter_config
            print('Using MSA shutter config: ', rate_object.meta.instrument.msa_metadata_file)

        # Run the stage 2 pipeline steps
        try:
            pipe_object = AssignWcsStep.call(rate_object)
            skip_file = False
        except:
            print("An error occured with AssignWcs. Likely: No open slits fall on detector", det)
            print("Skipping test for this file. \n")
            skip_file = True
       
        if not skip_file:
            msa_flagging_object = MSAFlagOpenStep.call(pipe_object)

            # Run the validation test
            %matplotlib inline

            result, result_msg, log_msgs = nptt.calwebb_spec2_pytests.auxiliary_code.msa_flagging_testing.run_msa_flagging_testing(
                                                                        msa_flagging_object, 
                                                                        msa_flagging_threshold = msa_flagging_threshold,
                                                                        rate_obj = rate_object,
                                                                        stellarity = stellarity, 
                                                                        operability_ref = operability_ref, 
                                                                        save_figs = save_figs, 
                                                                        show_figs = show_figs, 
                                                                        source_type = source_type, 
                                                                        debug = debug)

        else:
            result = 'skipped'


        # Did the test pass
        print("Did msa_flagging validation test pass? ", result, "\n\n")
        rd = {uncal_basename: result}
        results_dict.update(rd)
        
        # close all open files
        psutil.Process().open_files()
        closing_files = []
        for fd in psutil.Process().open_files():
            if data_dir.name in fd.path:
                closing_files.append(fd)
        for fd in closing_files:
            try:
                if verbose:
                    print('Closing file: ', fd)
                open(fd.fd).close()
            except:
                if verbose:
                    print('File already closed: ', fd)

Testing files for detector:  nrs1
Working with uncal_file:  /internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpln7a_gki/mos_g140m_line1_NRS1_uncal.fits

2022-10-06 00:37:13,963 - stpipe - INFO - PARS-DETECTOR1PIPELINE parameters found: /grp/crds/cache/references/jwst/jwst_nirspec_pars-detector1pipeline_0004.asdf
2022-10-06 00:37:13,995 - stpipe.Detector1Pipeline - INFO - Detector1Pipeline instance created.
2022-10-06 00:37:13,997 - stpipe.Detector1Pipeline.group_scale - INFO - GroupScaleStep instance created.
2022-10-06 00:37:13,998 - stpipe.Detector1Pipeline.dq_init - INFO - DQInitStep instance created.
2022-10-06 00:37:13,999 - stpipe.Detector1Pipeline.saturation - INFO - SaturationStep instance created.
2022-10-06 00:37:14,000 - stpipe.Detector1Pipeline.ipc - INFO - IPCStep instance created.
2022-10-06 00:37:14,001 - stpipe.Detector1Pipeline.superbias - INFO - SuperBiasStep instance created.
2022-10-06 00:37:14,002 - stpipe.Detector1Pipeline.refpix - INFO - RefPixStep instance created.
2022-10-06 00:37:14,004 - stpipe.Detector1Pipeline.rscd - INFO - RscdStep instance created.
2022-10-06 00:37:14,005 - stpipe.Detector1Pipeline.firstframe - INFO - FirstFrameStep instance created.
2022-10-06 00:37:14,006 - stpipe.Detector1Pipeline.lastframe - INFO - LastFrameStep instance created.
2022-10-06 00:37:14,007 - stpipe.Detector1Pipeline.linearity - INFO - LinearityStep instance created.
2022-10-06 00:37:14,008 - stpipe.Detector1Pipeline.dark_current - INFO - DarkCurrentStep instance created.
2022-10-06 00:37:14,010 - stpipe.Detector1Pipeline.reset - INFO - ResetStep instance created.
2022-10-06 00:37:14,011 - stpipe.Detector1Pipeline.persistence - INFO - PersistenceStep instance created.
2022-10-06 00:37:14,012 - stpipe.Detector1Pipeline.jump - INFO - JumpStep instance created.
2022-10-06 00:37:14,013 - stpipe.Detector1Pipeline.ramp_fit - INFO - RampFitStep instance created.
2022-10-06 00:37:14,014 - stpipe.Detector1Pipeline.gain_scale - INFO - GainScaleStep instance created.
2022-10-06 00:37:14,169 - stpipe.Detector1Pipeline - INFO - Step Detector1Pipeline running with args ('/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpln7a_gki/mos_g140m_line1_NRS1_uncal.fits',).
2022-10-06 00:37:14,179 - stpipe.Detector1Pipeline - INFO - Step Detector1Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_calibrated_ramp': False, 'steps': {'group_scale': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'dq_init': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'saturation': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'n_pix_grow_sat': 1}, 'ipc': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'superbias': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'refpix': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'odd_even_columns': True, 'use_side_ref_pixels': True, 'side_smoothing_length': 11, 'side_gain': 1.0, 'odd_even_rows': True}, 'rscd': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'type': 'baseline'}, 'firstframe': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'lastframe': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'linearity': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'dark_current': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'dark_output': None}, 'reset': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'persistence': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'input_trapsfilled': '', 'flag_pers_cutoff': 40.0, 'save_persistence': False, 'save_trapsfilled': True}, 'jump': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'rejection_threshold': 4.0, 'three_group_rejection_threshold': 6.0, 'four_group_rejection_threshold': 5.0, 'maximum_cores': 'none', 'flag_4_neighbors': True, 'max_jump_to_flag_neighbors': 1000.0, 'min_jump_to_flag_neighbors': 10.0, 'after_jump_flag_dn1': 0.0, 'after_jump_flag_time1': 0.0, 'after_jump_flag_dn2': 0.0, 'after_jump_flag_time2': 0.0}, 'ramp_fit': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'int_name': '', 'save_opt': False, 'opt_name': '', 'suppress_one_group': True, 'maximum_cores': 'none'}, 'gain_scale': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}}}
2022-10-06 00:37:14,366 - stpipe.Detector1Pipeline - INFO - Prefetching reference files for dataset: 'mos_g140m_line1_NRS1_uncal.fits' reftypes = ['dark', 'gain', 'linearity', 'mask', 'persat', 'readnoise', 'refpix', 'reset', 'rscd', 'saturation', 'superbias', 'trapdensity', 'trappars']
2022-10-06 00:37:14,377 - stpipe.Detector1Pipeline - INFO - Prefetch for DARK reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_dark_0140.fits'.
2022-10-06 00:37:14,379 - stpipe.Detector1Pipeline - INFO - Prefetch for GAIN reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_gain_0019.fits'.
2022-10-06 00:37:14,381 - stpipe.Detector1Pipeline - INFO - Prefetch for LINEARITY reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_linearity_0019.fits'.
2022-10-06 00:37:14,382 - stpipe.Detector1Pipeline - INFO - Prefetch for MASK reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_mask_0024.fits'.
2022-10-06 00:37:14,384 - stpipe.Detector1Pipeline - INFO - Prefetch for PERSAT reference file is 'N/A'.
2022-10-06 00:37:14,384 - stpipe.Detector1Pipeline - INFO - Prefetch for READNOISE reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_readnoise_0019.fits'.
2022-10-06 00:37:14,386 - stpipe.Detector1Pipeline - INFO - Prefetch for REFPIX reference file is 'N/A'.
2022-10-06 00:37:14,387 - stpipe.Detector1Pipeline - INFO - Prefetch for RESET reference file is 'N/A'.
2022-10-06 00:37:14,387 - stpipe.Detector1Pipeline - INFO - Prefetch for RSCD reference file is 'N/A'.
2022-10-06 00:37:14,388 - stpipe.Detector1Pipeline - INFO - Prefetch for SATURATION reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_saturation_0023.fits'.
2022-10-06 00:37:14,389 - stpipe.Detector1Pipeline - INFO - Prefetch for SUPERBIAS reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_superbias_0087.fits'.
2022-10-06 00:37:14,391 - stpipe.Detector1Pipeline - INFO - Prefetch for TRAPDENSITY reference file is 'N/A'.
2022-10-06 00:37:14,391 - stpipe.Detector1Pipeline - INFO - Prefetch for TRAPPARS reference file is 'N/A'.
2022-10-06 00:37:14,392 - stpipe.Detector1Pipeline - INFO - Starting calwebb_detector1 ...
2022-10-06 00:37:14,791 - stpipe.Detector1Pipeline.group_scale - INFO - Step group_scale running with args (<RampModel(1, 6, 2048, 2048) from mos_g140m_line1_NRS1_uncal.fits>,).
2022-10-06 00:37:14,793 - stpipe.Detector1Pipeline.group_scale - INFO - Step group_scale parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpln7a_gki'}
2022-10-06 00:37:14,929 - stpipe.Detector1Pipeline.group_scale - INFO - NFRAMES and FRMDIVSR are equal; correction not needed
2022-10-06 00:37:14,930 - stpipe.Detector1Pipeline.group_scale - INFO - Step will be skipped
2022-10-06 00:37:14,932 - stpipe.Detector1Pipeline.group_scale - INFO - Step group_scale done
2022-10-06 00:37:15,074 - stpipe.Detector1Pipeline.dq_init - INFO - Step dq_init running with args (<RampModel(1, 6, 2048, 2048) from mos_g140m_line1_NRS1_uncal.fits>,).
2022-10-06 00:37:15,076 - stpipe.Detector1Pipeline.dq_init - INFO - Step dq_init parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpln7a_gki'}
2022-10-06 00:37:15,099 - stpipe.Detector1Pipeline.dq_init - INFO - Using MASK reference file /grp/crds/cache/references/jwst/jwst_nirspec_mask_0024.fits
2022-10-06 00:37:15,510 - stpipe.Detector1Pipeline.dq_init - INFO - Step dq_init done
2022-10-06 00:37:15,647 - stpipe.Detector1Pipeline.saturation - INFO - Step saturation running with args (<RampModel(1, 6, 2048, 2048) from mos_g140m_line1_NRS1_uncal.fits>,).
2022-10-06 00:37:15,649 - stpipe.Detector1Pipeline.saturation - INFO - Step saturation parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpln7a_gki', 'n_pix_grow_sat': 1}
2022-10-06 00:37:15,672 - stpipe.Detector1Pipeline.saturation - INFO - Using SATURATION reference file /grp/crds/cache/references/jwst/jwst_nirspec_saturation_0023.fits
2022-10-06 00:37:17,459 - stpipe.Detector1Pipeline.saturation - INFO - Detected 108240 saturated pixels
2022-10-06 00:37:17,483 - stpipe.Detector1Pipeline.saturation - INFO - Detected 0 A/D floor pixels
2022-10-06 00:37:17,498 - stpipe.Detector1Pipeline.saturation - INFO - Step saturation done
2022-10-06 00:37:17,653 - stpipe.Detector1Pipeline.ipc - INFO - Step ipc running with args (<RampModel(1, 6, 2048, 2048) from mos_g140m_line1_NRS1_uncal.fits>,).
2022-10-06 00:37:17,654 - stpipe.Detector1Pipeline.ipc - INFO - Step ipc parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpln7a_gki'}
2022-10-06 00:37:17,655 - stpipe.Detector1Pipeline.ipc - INFO - Step skipped.
2022-10-06 00:37:17,658 - stpipe.Detector1Pipeline.ipc - INFO - Step ipc done
2022-10-06 00:37:17,802 - stpipe.Detector1Pipeline.superbias - INFO - Step superbias running with args (<RampModel(1, 6, 2048, 2048) from mos_g140m_line1_NRS1_uncal.fits>,).
2022-10-06 00:37:17,803 - stpipe.Detector1Pipeline.superbias - INFO - Step superbias parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpln7a_gki'}
2022-10-06 00:37:17,828 - stpipe.Detector1Pipeline.superbias - INFO - Using SUPERBIAS reference file /grp/crds/cache/references/jwst/jwst_nirspec_superbias_0087.fits
2022-10-06 00:37:18,359 - stpipe.Detector1Pipeline.superbias - INFO - Step superbias done
2022-10-06 00:37:18,509 - stpipe.Detector1Pipeline.refpix - INFO - Step refpix running with args (<RampModel(1, 6, 2048, 2048) from mos_g140m_line1_NRS1_uncal.fits>,).
2022-10-06 00:37:18,511 - stpipe.Detector1Pipeline.refpix - INFO - Step refpix parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpln7a_gki', 'odd_even_columns': True, 'use_side_ref_pixels': True, 'side_smoothing_length': 11, 'side_gain': 1.0, 'odd_even_rows': True}
2022-10-06 00:37:18,643 - stpipe.Detector1Pipeline.refpix - INFO - NIR full frame data
2022-10-06 00:37:18,644 - stpipe.Detector1Pipeline.refpix - INFO - The following parameters are valid for this mode:
2022-10-06 00:37:18,644 - stpipe.Detector1Pipeline.refpix - INFO - use_side_ref_pixels = True
2022-10-06 00:37:18,645 - stpipe.Detector1Pipeline.refpix - INFO - odd_even_columns = True
2022-10-06 00:37:18,645 - stpipe.Detector1Pipeline.refpix - INFO - side_smoothing_length = 11
2022-10-06 00:37:18,646 - stpipe.Detector1Pipeline.refpix - INFO - side_gain = 1.0
2022-10-06 00:37:18,646 - stpipe.Detector1Pipeline.refpix - INFO - The following parameter is not applicable and is ignored:
2022-10-06 00:37:18,646 - stpipe.Detector1Pipeline.refpix - INFO - odd_even_rows = False
2022-10-06 00:37:21,126 - stpipe.Detector1Pipeline.refpix - INFO - Step refpix done
2022-10-06 00:37:21,278 - stpipe.Detector1Pipeline.linearity - INFO - Step linearity running with args (<RampModel(1, 6, 2048, 2048) from mos_g140m_line1_NRS1_uncal.fits>,).
2022-10-06 00:37:21,280 - stpipe.Detector1Pipeline.linearity - INFO - Step linearity parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpln7a_gki'}
2022-10-06 00:37:21,304 - stpipe.Detector1Pipeline.linearity - INFO - Using Linearity reference file /grp/crds/cache/references/jwst/jwst_nirspec_linearity_0019.fits
2022-10-06 00:37:24,918 - stpipe.Detector1Pipeline.linearity - INFO - Step linearity done
2022-10-06 00:37:25,105 - stpipe.Detector1Pipeline.dark_current - INFO - Step dark_current running with args (<RampModel(1, 6, 2048, 2048) from mos_g140m_line1_NRS1_uncal.fits>,).
2022-10-06 00:37:25,106 - stpipe.Detector1Pipeline.dark_current - INFO - Step dark_current parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpln7a_gki', 'dark_output': None}
2022-10-06 00:37:25,130 - stpipe.Detector1Pipeline.dark_current - INFO - Using DARK reference file /grp/crds/cache/references/jwst/jwst_nirspec_dark_0140.fits
2022-10-06 00:37:26,464 - stpipe.Detector1Pipeline.dark_current - INFO - Science data nints=1, ngroups=6, nframes=1, groupgap=0
2022-10-06 00:37:26,465 - stpipe.Detector1Pipeline.dark_current - INFO - Dark data nints=1, ngroups=10, nframes=1, groupgap=0
2022-10-06 00:37:26,773 - stpipe.Detector1Pipeline.dark_current - INFO - Step dark_current done
2022-10-06 00:37:26,965 - stpipe.Detector1Pipeline.jump - INFO - Step jump running with args (<RampModel(1, 6, 2048, 2048) from mos_g140m_line1_NRS1_uncal.fits>,).
2022-10-06 00:37:26,967 - stpipe.Detector1Pipeline.jump - INFO - Step jump parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpln7a_gki', 'rejection_threshold': 4.0, 'three_group_rejection_threshold': 6.0, 'four_group_rejection_threshold': 5.0, 'maximum_cores': 'none', 'flag_4_neighbors': True, 'max_jump_to_flag_neighbors': 1000.0, 'min_jump_to_flag_neighbors': 10.0, 'after_jump_flag_dn1': 0.0, 'after_jump_flag_time1': 0.0, 'after_jump_flag_dn2': 0.0, 'after_jump_flag_time2': 0.0}
2022-10-06 00:37:26,977 - stpipe.Detector1Pipeline.jump - INFO - CR rejection threshold = 4 sigma
2022-10-06 00:37:26,991 - stpipe.Detector1Pipeline.jump - INFO - Using GAIN reference file: /grp/crds/cache/references/jwst/jwst_nirspec_gain_0019.fits
2022-10-06 00:37:27,155 - stpipe.Detector1Pipeline.jump - INFO - Using READNOISE reference file: /grp/crds/cache/references/jwst/jwst_nirspec_readnoise_0019.fits
2022-10-06 00:37:27,555 - stpipe.Detector1Pipeline.jump - INFO - Executing two-point difference method
2022-10-06 00:37:27,618 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 1:
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/jump/twopoint_difference.py:154: RuntimeWarning: All-NaN slice encountered
  max_ratio = np.nanmax(ratio, axis=0)
2022-10-06 00:37:29,818 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 41569 pixels with at least one CR from five or more groups.
2022-10-06 00:37:29,820 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 2712 pixels with at least one CR from four groups.
2022-10-06 00:37:29,820 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 3136 pixels with at least one CR from three groups.
2022-10-06 00:37:36,743 - stpipe.Detector1Pipeline.jump - INFO - Total elapsed time = 9.18645 sec
2022-10-06 00:37:36,747 - stpipe.Detector1Pipeline.jump - INFO - The execution time in seconds: 9.770020
2022-10-06 00:37:36,753 - stpipe.Detector1Pipeline.jump - INFO - Step jump done
2022-10-06 00:37:36,960 - stpipe.Detector1Pipeline.ramp_fit - INFO - Step ramp_fit running with args (<RampModel(1, 6, 2048, 2048) from mos_g140m_line1_NRS1_uncal.fits>,).
2022-10-06 00:37:36,961 - stpipe.Detector1Pipeline.ramp_fit - INFO - Step ramp_fit parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpln7a_gki', 'int_name': '', 'save_opt': False, 'opt_name': '', 'suppress_one_group': True, 'maximum_cores': 'none'}
2022-10-06 00:37:36,997 - stpipe.Detector1Pipeline.ramp_fit - INFO - Using READNOISE reference file: /grp/crds/cache/references/jwst/jwst_nirspec_readnoise_0019.fits
2022-10-06 00:37:36,997 - stpipe.Detector1Pipeline.ramp_fit - INFO - Using GAIN reference file: /grp/crds/cache/references/jwst/jwst_nirspec_gain_0019.fits
2022-10-06 00:37:37,112 - stpipe.Detector1Pipeline.ramp_fit - INFO - Using algorithm = ols
2022-10-06 00:37:37,113 - stpipe.Detector1Pipeline.ramp_fit - INFO - Using weighting = optimal
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
2022-10-06 00:38:01,959 - stpipe.Detector1Pipeline.ramp_fit - INFO - Number of groups per integration: 6
2022-10-06 00:38:01,960 - stpipe.Detector1Pipeline.ramp_fit - INFO - Number of integrations: 1
2022-10-06 00:38:02,139 - stpipe.Detector1Pipeline.ramp_fit - INFO - Step ramp_fit done
2022-10-06 00:38:02,379 - stpipe.Detector1Pipeline.gain_scale - INFO - Step gain_scale running with args (<ImageModel(2048, 2048) from mos_g140m_line1_NRS1_uncal.fits>,).
2022-10-06 00:38:02,380 - stpipe.Detector1Pipeline.gain_scale - INFO - Step gain_scale parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'gain_scale', 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpln7a_gki'}
2022-10-06 00:38:02,462 - stpipe.Detector1Pipeline.gain_scale - INFO - GAINFACT not found in gain reference file
2022-10-06 00:38:02,463 - stpipe.Detector1Pipeline.gain_scale - INFO - Step will be skipped
2022-10-06 00:38:02,468 - stpipe.Detector1Pipeline.gain_scale - INFO - Step gain_scale done
2022-10-06 00:38:02,613 - stpipe.Detector1Pipeline.gain_scale - INFO - Step gain_scale running with args (<CubeModel(1, 2048, 2048) from mos_g140m_line1_NRS1_uncal.fits>,).
2022-10-06 00:38:02,614 - stpipe.Detector1Pipeline.gain_scale - INFO - Step gain_scale parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'gain_scaleints', 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpln7a_gki'}
2022-10-06 00:38:02,683 - stpipe.Detector1Pipeline.gain_scale - INFO - GAINFACT not found in gain reference file
2022-10-06 00:38:02,684 - stpipe.Detector1Pipeline.gain_scale - INFO - Step will be skipped
2022-10-06 00:38:02,688 - stpipe.Detector1Pipeline.gain_scale - INFO - Step gain_scale done
2022-10-06 00:38:02,689 - stpipe.Detector1Pipeline - INFO - ... ending calwebb_detector1
2022-10-06 00:38:02,689 - stpipe.Detector1Pipeline - INFO - Results used CRDS context: jwst_0988.pmap
2022-10-06 00:38:02,690 - stpipe.Detector1Pipeline - INFO - Step Detector1Pipeline done
2022-10-06 00:38:02,699 - stpipe.AssignWcsStep - INFO - AssignWcsStep instance created.
2022-10-06 00:38:02,842 - stpipe.AssignWcsStep - INFO - Step AssignWcsStep running with args (<ImageModel(2048, 2048) from mos_g140m_line1_NRS1_uncal.fits>,).
2022-10-06 00:38:02,843 - stpipe.AssignWcsStep - INFO - Step AssignWcsStep parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'sip_approx': True, 'sip_max_pix_error': 0.25, 'sip_degree': None, 'sip_max_inv_pix_error': 0.25, 'sip_inv_degree': None, 'sip_npoints': 32, 'slit_y_low': -0.55, 'slit_y_high': 0.55}

Using MSA shutter config:  /internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpln7a_gki/V8460001000101_msa.fits

2022-10-06 00:38:03,004 - stpipe.AssignWcsStep - INFO - Retrieving open MSA slitlets for msa_metadata_id = 1 and dither_index = 1
2022-10-06 00:38:03,226 - stpipe.AssignWcsStep - INFO - gwa_ytilt is 0.1260581910610199 deg
2022-10-06 00:38:03,226 - stpipe.AssignWcsStep - INFO - gwa_xtilt is 0.3316612243652344 deg
2022-10-06 00:38:03,227 - stpipe.AssignWcsStep - INFO - theta_y correction: 0.0002496099796717191 deg
2022-10-06 00:38:03,229 - stpipe.AssignWcsStep - INFO - theta_x correction: 0.0 deg
2022-10-06 00:38:03,468 - stpipe.AssignWcsStep - INFO - Removing slit 6 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,481 - stpipe.AssignWcsStep - INFO - Removing slit 7 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,493 - stpipe.AssignWcsStep - INFO - Removing slit 8 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,505 - stpipe.AssignWcsStep - INFO - Removing slit 9 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,517 - stpipe.AssignWcsStep - INFO - Removing slit 10 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,529 - stpipe.AssignWcsStep - INFO - Removing slit 11 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,542 - stpipe.AssignWcsStep - INFO - Removing slit 14 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,554 - stpipe.AssignWcsStep - INFO - Removing slit 15 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,566 - stpipe.AssignWcsStep - INFO - Removing slit 16 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,578 - stpipe.AssignWcsStep - INFO - Removing slit 17 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,590 - stpipe.AssignWcsStep - INFO - Removing slit 18 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,602 - stpipe.AssignWcsStep - INFO - Removing slit 19 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,614 - stpipe.AssignWcsStep - INFO - Removing slit 21 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,626 - stpipe.AssignWcsStep - INFO - Removing slit 25 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,639 - stpipe.AssignWcsStep - INFO - Removing slit 26 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,651 - stpipe.AssignWcsStep - INFO - Removing slit 27 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,663 - stpipe.AssignWcsStep - INFO - Removing slit 29 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,675 - stpipe.AssignWcsStep - INFO - Removing slit 31 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,687 - stpipe.AssignWcsStep - INFO - Removing slit 32 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,699 - stpipe.AssignWcsStep - INFO - Removing slit 33 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,711 - stpipe.AssignWcsStep - INFO - Removing slit 35 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,724 - stpipe.AssignWcsStep - INFO - Removing slit 37 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,736 - stpipe.AssignWcsStep - INFO - Removing slit 38 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,748 - stpipe.AssignWcsStep - INFO - Removing slit 39 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,760 - stpipe.AssignWcsStep - INFO - Removing slit 41 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,795 - stpipe.AssignWcsStep - INFO - Removing slit 1 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,806 - stpipe.AssignWcsStep - INFO - Removing slit 2 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,819 - stpipe.AssignWcsStep - INFO - Removing slit 3 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,831 - stpipe.AssignWcsStep - INFO - Removing slit 4 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,843 - stpipe.AssignWcsStep - INFO - Removing slit 5 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,855 - stpipe.AssignWcsStep - INFO - Removing slit 12 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,867 - stpipe.AssignWcsStep - INFO - Removing slit 13 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,879 - stpipe.AssignWcsStep - INFO - Removing slit 20 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,891 - stpipe.AssignWcsStep - INFO - Removing slit 22 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,903 - stpipe.AssignWcsStep - INFO - Removing slit 23 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,915 - stpipe.AssignWcsStep - INFO - Removing slit 24 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,927 - stpipe.AssignWcsStep - INFO - Removing slit 28 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,939 - stpipe.AssignWcsStep - INFO - Removing slit 30 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,951 - stpipe.AssignWcsStep - INFO - Removing slit 34 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,963 - stpipe.AssignWcsStep - INFO - Removing slit 36 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:03,975 - stpipe.AssignWcsStep - INFO - Removing slit 40 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:04,261 - stpipe.AssignWcsStep - INFO - Slits projected on detector NRS1: [42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68]
2022-10-06 00:38:04,262 - stpipe.AssignWcsStep - INFO - Computing WCS for 27 open slitlets
2022-10-06 00:38:04,298 - stpipe.AssignWcsStep - INFO - gwa_ytilt is 0.1260581910610199 deg
2022-10-06 00:38:04,298 - stpipe.AssignWcsStep - INFO - gwa_xtilt is 0.3316612243652344 deg
2022-10-06 00:38:04,299 - stpipe.AssignWcsStep - INFO - theta_y correction: 0.0002496099796717191 deg
2022-10-06 00:38:04,300 - stpipe.AssignWcsStep - INFO - theta_x correction: 0.0 deg
2022-10-06 00:38:04,313 - stpipe.AssignWcsStep - INFO - SPORDER= -1, wrange=[9.7e-07, 1.89e-06]
2022-10-06 00:38:04,458 - stpipe.AssignWcsStep - INFO - There are 2 open slits in quadrant 1
2022-10-06 00:38:04,474 - stpipe.AssignWcsStep - INFO - There are 11 open slits in quadrant 2
2022-10-06 00:38:04,557 - stpipe.AssignWcsStep - INFO - There are 4 open slits in quadrant 3
2022-10-06 00:38:04,588 - stpipe.AssignWcsStep - INFO - There are 10 open slits in quadrant 4
2022-10-06 00:38:04,833 - stpipe.AssignWcsStep - INFO - There are 0 open slits in quadrant 5
2022-10-06 00:38:05,000 - stpipe.AssignWcsStep - INFO - Created a NIRSPEC nrs_msaspec pipeline with references {'distortion': None, 'filteroffset': None, 'specwcs': None, 'regions': None, 'wavelengthrange': '/grp/crds/cache/references/jwst/jwst_nirspec_wavelengthrange_0004.asdf', 'camera': '/grp/crds/cache/references/jwst/jwst_nirspec_camera_0004.asdf', 'collimator': '/grp/crds/cache/references/jwst/jwst_nirspec_collimator_0004.asdf', 'disperser': '/grp/crds/cache/references/jwst/jwst_nirspec_disperser_0032.asdf', 'fore': '/grp/crds/cache/references/jwst/jwst_nirspec_fore_0023.asdf', 'fpa': '/grp/crds/cache/references/jwst/jwst_nirspec_fpa_0005.asdf', 'msa': '/grp/crds/cache/references/jwst/jwst_nirspec_msa_0005.asdf', 'ote': '/grp/crds/cache/references/jwst/jwst_nirspec_ote_0005.asdf', 'ifupost': None, 'ifufore': None, 'ifuslicer': None, 'msametafile': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmpln7a_gki/V8460001000101_msa.fits'}
2022-10-06 00:38:05,878 - stpipe.AssignWcsStep - INFO - COMPLETED assign_wcs
2022-10-06 00:38:05,892 - stpipe.AssignWcsStep - INFO - Results used CRDS context: jwst_0988.pmap
2022-10-06 00:38:05,893 - stpipe.AssignWcsStep - INFO - Step AssignWcsStep done
2022-10-06 00:38:05,903 - stpipe.MSAFlagOpenStep - INFO - MSAFlagOpenStep instance created.
2022-10-06 00:38:06,089 - stpipe.MSAFlagOpenStep - INFO - Step MSAFlagOpenStep running with args (<ImageModel(2048, 2048) from mos_g140m_line1_NRS1_uncal.fits>,).
2022-10-06 00:38:06,091 - stpipe.MSAFlagOpenStep - INFO - Step MSAFlagOpenStep parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}
2022-10-06 00:38:06,108 - stpipe.MSAFlagOpenStep - INFO - Using reference file /grp/crds/cache/references/jwst/jwst_nirspec_msaoper_0001.json
2022-10-06 00:38:06,109 - stpipe.JwstStep - INFO - JwstStep instance created.
2022-10-06 00:38:06,245 - stpipe.MSAFlagOpenStep - INFO - gwa_ytilt is 0.1260581910610199 deg
2022-10-06 00:38:06,246 - stpipe.MSAFlagOpenStep - INFO - gwa_xtilt is 0.3316612243652344 deg
2022-10-06 00:38:06,247 - stpipe.MSAFlagOpenStep - INFO - theta_y correction: 0.0002496099796717191 deg
2022-10-06 00:38:06,248 - stpipe.MSAFlagOpenStep - INFO - theta_x correction: 0.0 deg
2022-10-06 00:38:06,261 - stpipe.MSAFlagOpenStep - INFO - SPORDER= -1, wrange=[9.7e-07, 1.89e-06]
2022-10-06 00:38:06,406 - stpipe.MSAFlagOpenStep - INFO - There are 5 open slits in quadrant 1
2022-10-06 00:38:06,443 - stpipe.MSAFlagOpenStep - INFO - There are 3 open slits in quadrant 2
2022-10-06 00:38:06,466 - stpipe.MSAFlagOpenStep - INFO - There are 9 open slits in quadrant 3
2022-10-06 00:38:06,534 - stpipe.MSAFlagOpenStep - INFO - There are 3 open slits in quadrant 4
2022-10-06 00:38:06,557 - stpipe.MSAFlagOpenStep - INFO - There are 0 open slits in quadrant 5
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/astropy/modeling/bounding_box.py:83: RuntimeWarning: Invalid interval: upper bound 2047.5 is strictly less than lower bound 2324.820049828629.
  warnings.warn(f"Invalid interval: upper bound {upper} "
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/astropy/modeling/bounding_box.py:83: RuntimeWarning: Invalid interval: upper bound 2047.5 is strictly less than lower bound 2244.599990785436.
  warnings.warn(f"Invalid interval: upper bound {upper} "
2022-10-06 00:38:18,687 - stpipe.MSAFlagOpenStep - INFO - Results used CRDS context: jwst_0988.pmap
2022-10-06 00:38:18,688 - stpipe.MSAFlagOpenStep - INFO - Step MSAFlagOpenStep done

2022-10-06 00:38:19,601 - stpipe.AssignWcsStep - INFO - AssignWcsStep instance created.
2022-10-06 00:38:19,612 - stpipe.AssignWcsStep - INFO - AssignWcsStep instance created.
2022-10-06 00:38:19,866 - stpipe.AssignWcsStep - INFO - Step AssignWcsStep running with args (<ImageModel(2048, 2048) from mos_g140m_line1_NRS1_uncal.fits>,).
2022-10-06 00:38:19,867 - stpipe.AssignWcsStep - INFO - Step AssignWcsStep parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'sip_approx': True, 'sip_max_pix_error': 0.25, 'sip_degree': None, 'sip_max_inv_pix_error': 0.25, 'sip_inv_degree': None, 'sip_npoints': 32, 'slit_y_low': -0.45, 'slit_y_high': 0.45}
2022-10-06 00:38:20,012 - stpipe.AssignWcsStep - INFO - Retrieving open MSA slitlets for msa_metadata_id = 1 and dither_index = 1
2022-10-06 00:38:20,070 - stpipe.AssignWcsStep - INFO - gwa_ytilt is 0.1260581910610199 deg
2022-10-06 00:38:20,071 - stpipe.AssignWcsStep - INFO - gwa_xtilt is 0.3316612243652344 deg
2022-10-06 00:38:20,071 - stpipe.AssignWcsStep - INFO - theta_y correction: 0.0002496099796717191 deg
2022-10-06 00:38:20,073 - stpipe.AssignWcsStep - INFO - theta_x correction: 0.0 deg
2022-10-06 00:38:20,259 - stpipe.AssignWcsStep - INFO - Removing slit 1 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:20,272 - stpipe.AssignWcsStep - INFO - Removing slit 2 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:20,323 - stpipe.AssignWcsStep - INFO - Removing slit 6 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-10-06 00:38:20,481 - stpipe.AssignWcsStep - INFO - Slits projected on detector NRS1: [3, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
2022-10-06 00:38:20,482 - stpipe.AssignWcsStep - INFO - Computing WCS for 16 open slitlets
2022-10-06 00:38:20,519 - stpipe.AssignWcsStep - INFO - gwa_ytilt is 0.1260581910610199 deg
2022-10-06 00:38:20,519 - stpipe.AssignWcsStep - INFO - gwa_xtilt is 0.3316612243652344 deg
2022-10-06 00:38:20,520 - stpipe.AssignWcsStep - INFO - theta_y correction: 0.0002496099796717191 deg
2022-10-06 00:38:20,521 - stpipe.AssignWcsStep - INFO - theta_x correction: 0.0 deg
2022-10-06 00:38:20,534 - stpipe.AssignWcsStep - INFO - SPORDER= -1, wrange=[9.7e-07, 1.89e-06]
2022-10-06 00:38:20,682 - stpipe.AssignWcsStep - INFO - There are 3 open slits in quadrant 1
2022-10-06 00:38:20,706 - stpipe.AssignWcsStep - INFO - There are 2 open slits in quadrant 2
2022-10-06 00:38:20,721 - stpipe.AssignWcsStep - INFO - There are 9 open slits in quadrant 3
2022-10-06 00:38:20,791 - stpipe.AssignWcsStep - INFO - There are 2 open slits in quadrant 4
2022-10-06 00:38:20,808 - stpipe.AssignWcsStep - INFO - There are 0 open slits in quadrant 5
2022-10-06 00:38:20,965 - stpipe.AssignWcsStep - INFO - Created a NIRSPEC nrs_msaspec pipeline with references {'distortion': None, 'filteroffset': None, 'specwcs': None, 'regions': None, 'wavelengthrange': '/grp/crds/cache/references/jwst/jwst_nirspec_wavelengthrange_0004.asdf', 'camera': '/grp/crds/cache/references/jwst/jwst_nirspec_camera_0004.asdf', 'collimator': '/grp/crds/cache/references/jwst/jwst_nirspec_collimator_0004.asdf', 'disperser': '/grp/crds/cache/references/jwst/jwst_nirspec_disperser_0032.asdf', 'fore': '/grp/crds/cache/references/jwst/jwst_nirspec_fore_0023.asdf', 'fpa': '/grp/crds/cache/references/jwst/jwst_nirspec_fpa_0005.asdf', 'msa': '/grp/crds/cache/references/jwst/jwst_nirspec_msa_0005.asdf', 'ote': '/grp/crds/cache/references/jwst/jwst_nirspec_ote_0005.asdf', 'ifupost': None, 'ifufore': None, 'ifuslicer': None, 'msametafile': 'fopens_metafile_msa.fits'}
2022-10-06 00:38:21,645 - stpipe.AssignWcsStep - INFO - COMPLETED assign_wcs
2022-10-06 00:38:21,660 - stpipe.AssignWcsStep - INFO - Results used CRDS context: jwst_0988.pmap
2022-10-06 00:38:21,660 - stpipe.AssignWcsStep - INFO - Step AssignWcsStep done

Working with slit/slice:  3
Slit min and max in y direction:  -0.5 0.5
bounding box rounded to next integer:  [[1677 2048]
 [1630 1640]]
Max value in slity array (ignoring NANs):  0.49983590702157094
percentage of F/O trace that was flagged: 99.8

Working with slit/slice:  4
Slit min and max in y direction:  -0.5 0.5
bounding box rounded to next integer:  [[1628 2048]
 [1320 1330]]
Max value in slity array (ignoring NANs):  0.4999416262418918
percentage of F/O trace that was flagged: 99.8

Working with slit/slice:  5
Slit min and max in y direction:  -0.5 0.5
bounding box rounded to next integer:  [[1536 2048]
 [1592 1602]]
Max value in slity array (ignoring NANs):  0.499395872247531
percentage of F/O trace that was flagged: 100.0

Working with slit/slice:  7
Slit min and max in y direction:  -0.5 0.5
bounding box rounded to next integer:  [[1531 2048]
 [ 240  250]]
Max value in slity array (ignoring NANs):  0.49982577845792203
percentage of F/O trace that was flagged: 100.0

Working with slit/slice:  8
Slit min and max in y direction:  -0.5 0.5
bounding box rounded to next integer:  [[1510 2048]
 [ 164  174]]
Max value in slity array (ignoring NANs):  0.4999363264129128
percentage of F/O trace that was flagged: 99.8


# Quickly see if the test passed: Do the NIRSpec implementation and the pipeline's agree within <= 99.5%?

print('These are the final results of the tests: ')
for key, val in results_dict.items():
    if not isinstance(val, str):
        if val:
            val = 'PASSED'
        else:
            val = 'FAILED'
    print('{:<40} {:<8}'.format(key, val))

These are the final results of the tests: 
mos_g140m_line1_NRS1_uncal.fits          PASSED  
mos_g140m_line1_NRS2_uncal.fits          PASSED

JWST Pipeline Validation Testing Notebook: spec2, msa_flagging step¶

Table of Contents¶

Imports¶

Test Description¶

Results¶

Calibration WG Requested Algorithm:¶

Defining Terms¶

Run the JWST pipeline and msa_flagging validation tests¶

Testing Data Set¶

¶

About this Notebook¶